{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('/home/luke/Desktop/kaggle/Home_Credit_Default_Risk')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_test = pd.read_csv('application_test.csv')\n",
    "application_train = pd.read_csv('application_train.csv')\n",
    "merged_df = pd.read_csv('processed_input_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = merged_df['TARGET']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_unique_high_corr_feature(df,first_n_features):\n",
    "    \n",
    "    #Geting sorted list of contributio of TARGET\n",
    "    correlations = df.corr()['TARGET'].sort_values(ascending=False)\n",
    "    \n",
    "    abs_corr = abs(correlations).sort_values(ascending=False)\n",
    "    \n",
    "    #Many of the features contain(Max, Min and Mean) contribute the same, so dedup\n",
    "    \n",
    "    corr_top_unique_features = abs_corr.head(first_n_features).drop_duplicates()\n",
    "    \n",
    "    #Return a list of distince features contribute to the model\n",
    "    \n",
    "    return np.array(corr_top_unique_features.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_list = get_unique_high_corr_feature(merged_df,600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_high_corr_data = merged_df[name_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def missing_values_tables(df):\n",
    "    \n",
    "    mis_val = df.isnull().sum()\n",
    "    mis_val_percent = 100*df.isnull().sum()/len(df)\n",
    "    mis_val_table = pd.concat([mis_val,mis_val_percent],axis = 1)\n",
    "    \n",
    "    mis_val_table_ren_columns = mis_val_table.rename(\n",
    "    columns = {0:'Missing Values',1:'% of Total Values'})\n",
    "    \n",
    "    mis_val_table_ren_columns = mis_val_table_ren_columns[\n",
    "        mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n",
    "    '% of Total Values',ascending=False).round(1)\n",
    "    \n",
    "    #print('Your selected dataframe has '+str(df.shape[1])+\" columns.\\n\" \"There are \" + str(mis_val_table_ren_columns.shape[0])+\n",
    "    #    ' column that have missing values.')\n",
    "    return mis_val_table_ren_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#missing_values_tables(new_high_corr_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_toomuchmissing_columns(df,cutoff_percent):\n",
    "    thresholds_list = missing_values_tables(df)['% of Total Values']<=cutoff_percent\n",
    "    thresholds_list = thresholds_list[thresholds_list==True]\n",
    "    \n",
    "    maintain_list = np.array(thresholds_list.index)\n",
    "    \n",
    "    return df[maintain_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_data = drop_toomuchmissing_columns(new_high_corr_data,70)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Label Encoding\n",
    "def process_dataframe(input_df, encoder_dict=None):\n",
    "    \"\"\" Process a dataframe into a form useable by LightGBM \"\"\"\n",
    "\n",
    "    # Label encode categoricals\n",
    "    print('Label encoding categorical features...')\n",
    "    categorical_feats = input_df.columns[input_df.dtypes == 'object']\n",
    "    for feat in categorical_feats:\n",
    "        encoder = LabelEncoder()\n",
    "        input_df[feat] = encoder.fit_transform(input_df[feat].fillna('NULL'))\n",
    "    print('Label encoding complete.')\n",
    "\n",
    "    return input_df, categorical_feats.tolist(), encoder_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label encoding categorical features...\n",
      "Label encoding complete.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(        TARGET  EXT_SOURCE_3  EXT_SOURCE_2  EXT_SOURCE_1  \\\n",
       " 0          1.0      0.139376      0.262949      0.083037   \n",
       " 1          0.0           NaN      0.622246      0.311267   \n",
       " 2          0.0      0.729567      0.555912           NaN   \n",
       " 3          0.0           NaN      0.650442           NaN   \n",
       " 4          0.0           NaN      0.322738           NaN   \n",
       " 5          0.0      0.621226      0.354225           NaN   \n",
       " 6          0.0      0.492060      0.724000      0.774761   \n",
       " 7          0.0      0.540654      0.714279           NaN   \n",
       " 8          0.0      0.751724      0.205747      0.587334   \n",
       " 9          0.0           NaN      0.746644           NaN   \n",
       " 10         0.0      0.363945      0.651862      0.319760   \n",
       " 11         0.0      0.652897      0.555183      0.722044   \n",
       " 12         0.0      0.176653      0.715042      0.464831   \n",
       " 13         0.0      0.770087      0.566907           NaN   \n",
       " 14         0.0           NaN      0.642656      0.721940   \n",
       " 15         0.0      0.678568      0.346634      0.115634   \n",
       " 16         0.0      0.062103      0.236378           NaN   \n",
       " 17         0.0           NaN      0.683513           NaN   \n",
       " 18         0.0      0.556727      0.706428           NaN   \n",
       " 19         0.0      0.477649      0.586617           NaN   \n",
       " 20         0.0           NaN      0.113375      0.565655   \n",
       " 21         0.0      0.542445      0.233767      0.437709   \n",
       " 22         0.0      0.358951      0.457143           NaN   \n",
       " 23         0.0      0.669057      0.624305           NaN   \n",
       " 24         0.0      0.565608      0.786179           NaN   \n",
       " 25         0.0      0.461482      0.651406      0.561948   \n",
       " 26         1.0      0.190706      0.548477           NaN   \n",
       " 27         0.0      0.659406      0.541124           NaN   \n",
       " 28         0.0      0.524496      0.685011      0.600396   \n",
       " 29         0.0           NaN      0.502779      0.297914   \n",
       " ...        ...           ...           ...           ...   \n",
       " 356225     NaN      0.627991      0.544646      0.623896   \n",
       " 356226     NaN      0.659406      0.654740      0.554915   \n",
       " 356227     NaN           NaN      0.008777      0.361450   \n",
       " 356228     NaN      0.483050      0.549354      0.725281   \n",
       " 356229     NaN      0.614414      0.540945           NaN   \n",
       " 356230     NaN      0.698668      0.350273      0.498882   \n",
       " 356231     NaN      0.612704      0.554481      0.587915   \n",
       " 356232     NaN      0.417100      0.664593      0.204548   \n",
       " 356233     NaN           NaN      0.385043           NaN   \n",
       " 356234     NaN      0.073965      0.689392      0.448657   \n",
       " 356235     NaN      0.295583      0.387059      0.478887   \n",
       " 356236     NaN      0.687933      0.095077      0.642719   \n",
       " 356237     NaN      0.508287      0.565268      0.563175   \n",
       " 356238     NaN      0.631355      0.287920      0.574515   \n",
       " 356239     NaN      0.556727      0.635018      0.787271   \n",
       " 356240     NaN           NaN      0.590742           NaN   \n",
       " 356241     NaN      0.459690      0.408147      0.440056   \n",
       " 356242     NaN      0.733815      0.432684      0.573130   \n",
       " 356243     NaN      0.415347      0.210374      0.527067   \n",
       " 356244     NaN      0.834784      0.347457      0.363857   \n",
       " 356245     NaN      0.569149      0.472553      0.410389   \n",
       " 356246     NaN           NaN      0.465881           NaN   \n",
       " 356247     NaN      0.631355      0.471719      0.851722   \n",
       " 356248     NaN      0.255332      0.689832      0.442558   \n",
       " 356249     NaN      0.240541      0.762352      0.174671   \n",
       " 356250     NaN      0.643026      0.648575           NaN   \n",
       " 356251     NaN           NaN      0.684596           NaN   \n",
       " 356252     NaN      0.283712      0.632770      0.733503   \n",
       " 356253     NaN      0.595456      0.445701      0.373090   \n",
       " 356254     NaN      0.272134      0.456541           NaN   \n",
       " \n",
       "         CNT_DRAWINGS_ATM_CURRENT_CCMEAN  DAYS_FIRST_DRAWING_PRVMAX  \\\n",
       " 0                                   NaN                        NaN   \n",
       " 1                                   NaN                        NaN   \n",
       " 2                                   NaN                        NaN   \n",
       " 3                                   NaN                        NaN   \n",
       " 4                                   NaN                        NaN   \n",
       " 5                                   NaN                        NaN   \n",
       " 6                                   NaN                        NaN   \n",
       " 7                                   NaN                        NaN   \n",
       " 8                              0.013670                    -2277.0   \n",
       " 9                                   NaN                        NaN   \n",
       " 10                                  NaN                        NaN   \n",
       " 11                                  NaN                        NaN   \n",
       " 12                                  NaN                        NaN   \n",
       " 13                                  NaN                        NaN   \n",
       " 14                                  NaN                        NaN   \n",
       " 15                                  NaN                        NaN   \n",
       " 16                                  NaN                        NaN   \n",
       " 17                                  NaN                        NaN   \n",
       " 18                                  NaN                        NaN   \n",
       " 19                                  NaN                        NaN   \n",
       " 20                                  NaN                        NaN   \n",
       " 21                                  NaN                        NaN   \n",
       " 22                                  NaN                        NaN   \n",
       " 23                                  NaN                        NaN   \n",
       " 24                                  NaN                        NaN   \n",
       " 25                                  NaN                        NaN   \n",
       " 26                                  NaN                        NaN   \n",
       " 27                                  NaN                        NaN   \n",
       " 28                                  NaN                        NaN   \n",
       " 29                                  NaN                        NaN   \n",
       " ...                                 ...                        ...   \n",
       " 356225                              NaN                        NaN   \n",
       " 356226                              NaN                        NaN   \n",
       " 356227                              NaN                        NaN   \n",
       " 356228                              NaN                        NaN   \n",
       " 356229                              NaN                        NaN   \n",
       " 356230                              NaN                        NaN   \n",
       " 356231                              NaN                        NaN   \n",
       " 356232                              NaN                        NaN   \n",
       " 356233                              NaN                        NaN   \n",
       " 356234                              NaN                        NaN   \n",
       " 356235                              NaN                        NaN   \n",
       " 356236                         0.039907                    -2398.0   \n",
       " 356237                              NaN                        NaN   \n",
       " 356238                              NaN                        NaN   \n",
       " 356239                         0.031134                    -2854.0   \n",
       " 356240                              NaN                        NaN   \n",
       " 356241                              NaN                        NaN   \n",
       " 356242                              NaN                        NaN   \n",
       " 356243                              NaN                        NaN   \n",
       " 356244                              NaN                        NaN   \n",
       " 356245                              NaN                        NaN   \n",
       " 356246                              NaN                        NaN   \n",
       " 356247                              NaN                        NaN   \n",
       " 356248                         0.061426                     -704.0   \n",
       " 356249                              NaN                        NaN   \n",
       " 356250                              NaN                        NaN   \n",
       " 356251                              NaN                        NaN   \n",
       " 356252                              NaN                        NaN   \n",
       " 356253                              NaN                        NaN   \n",
       " 356254                         0.217273                     -373.0   \n",
       " \n",
       "         AMT_INST_MIN_REGULARITY  DAYS_FIRST_DRAWING  \\\n",
       " 0                           NaN                 NaN   \n",
       " 1                           NaN                 NaN   \n",
       " 2                           NaN                 NaN   \n",
       " 3                      0.000000                 NaN   \n",
       " 4                           NaN                 NaN   \n",
       " 5                           NaN                 NaN   \n",
       " 6                           NaN                 NaN   \n",
       " 7                           NaN                 NaN   \n",
       " 8                           NaN             -2277.0   \n",
       " 9                           NaN                 NaN   \n",
       " 10                          NaN                 NaN   \n",
       " 11                          NaN                 NaN   \n",
       " 12                          NaN                 NaN   \n",
       " 13                          NaN                 NaN   \n",
       " 14                          NaN                 NaN   \n",
       " 15                          NaN                 NaN   \n",
       " 16                          NaN                 NaN   \n",
       " 17                     0.000000                 NaN   \n",
       " 18                          NaN                 NaN   \n",
       " 19                     0.000000                 NaN   \n",
       " 20                          NaN                 NaN   \n",
       " 21                          NaN                 NaN   \n",
       " 22                          NaN                 NaN   \n",
       " 23                          NaN                 NaN   \n",
       " 24                          NaN                 NaN   \n",
       " 25                          NaN                 NaN   \n",
       " 26                          NaN                 NaN   \n",
       " 27                          NaN                 NaN   \n",
       " 28                          NaN                 NaN   \n",
       " 29                          NaN                 NaN   \n",
       " ...                         ...                 ...   \n",
       " 356225                      NaN                 NaN   \n",
       " 356226                      NaN                 NaN   \n",
       " 356227                      NaN                 NaN   \n",
       " 356228                      NaN                 NaN   \n",
       " 356229                      NaN                 NaN   \n",
       " 356230                 0.000000                 NaN   \n",
       " 356231                      NaN                 NaN   \n",
       " 356232                      NaN                 NaN   \n",
       " 356233                      NaN                 NaN   \n",
       " 356234                      NaN                 NaN   \n",
       " 356235                      NaN                 NaN   \n",
       " 356236                      NaN             -2398.0   \n",
       " 356237                 0.000000                 NaN   \n",
       " 356238                      NaN                 NaN   \n",
       " 356239                      NaN             -2854.0   \n",
       " 356240                      NaN                 NaN   \n",
       " 356241                      NaN                 NaN   \n",
       " 356242                      NaN                 NaN   \n",
       " 356243                      NaN                 NaN   \n",
       " 356244                      NaN                 NaN   \n",
       " 356245                 0.000000                 NaN   \n",
       " 356246                      NaN                 NaN   \n",
       " 356247                      NaN                 NaN   \n",
       " 356248              6963.373634              -704.0   \n",
       " 356249                      NaN                 NaN   \n",
       " 356250                      NaN                 NaN   \n",
       " 356251                      NaN                 NaN   \n",
       " 356252                      NaN                 NaN   \n",
       " 356253                      NaN                 NaN   \n",
       " 356254              8208.649921              -373.0   \n",
       " \n",
       "         DAYS_FIRST_DRAWING_PRVMIN  AMT_BALANCE_CCMIN  \\\n",
       " 0                             NaN                NaN   \n",
       " 1                             NaN                NaN   \n",
       " 2                             NaN                NaN   \n",
       " 3                             NaN           0.000000   \n",
       " 4                             NaN                NaN   \n",
       " 5                             NaN                NaN   \n",
       " 6                             NaN                NaN   \n",
       " 7                             NaN                NaN   \n",
       " 8                         -2277.0       16910.295179   \n",
       " 9                             NaN                NaN   \n",
       " 10                            NaN                NaN   \n",
       " 11                            NaN                NaN   \n",
       " 12                            NaN                NaN   \n",
       " 13                            NaN                NaN   \n",
       " 14                            NaN                NaN   \n",
       " 15                            NaN                NaN   \n",
       " 16                            NaN                NaN   \n",
       " 17                            NaN           0.000000   \n",
       " 18                            NaN                NaN   \n",
       " 19                            NaN           0.000000   \n",
       " 20                            NaN                NaN   \n",
       " 21                            NaN                NaN   \n",
       " 22                            NaN                NaN   \n",
       " 23                            NaN                NaN   \n",
       " 24                            NaN                NaN   \n",
       " 25                            NaN                NaN   \n",
       " 26                            NaN                NaN   \n",
       " 27                            NaN                NaN   \n",
       " 28                            NaN                NaN   \n",
       " 29                            NaN                NaN   \n",
       " ...                           ...                ...   \n",
       " 356225                        NaN                NaN   \n",
       " 356226                        NaN                NaN   \n",
       " 356227                        NaN                NaN   \n",
       " 356228                        NaN                NaN   \n",
       " 356229                        NaN                NaN   \n",
       " 356230                        NaN           0.000000   \n",
       " 356231                        NaN                NaN   \n",
       " 356232                        NaN                NaN   \n",
       " 356233                        NaN                NaN   \n",
       " 356234                        NaN                NaN   \n",
       " 356235                        NaN                NaN   \n",
       " 356236                    -2398.0       16192.733896   \n",
       " 356237                        NaN           0.000000   \n",
       " 356238                        NaN                NaN   \n",
       " 356239                    -2854.0       15758.470496   \n",
       " 356240                        NaN                NaN   \n",
       " 356241                        NaN                NaN   \n",
       " 356242                        NaN                NaN   \n",
       " 356243                        NaN                NaN   \n",
       " 356244                        NaN                NaN   \n",
       " 356245                        NaN           0.000000   \n",
       " 356246                        NaN                NaN   \n",
       " 356247                        NaN                NaN   \n",
       " 356248                     -704.0      136954.442970   \n",
       " 356249                        NaN                NaN   \n",
       " 356250                        NaN                NaN   \n",
       " 356251                        NaN                NaN   \n",
       " 356252                        NaN                NaN   \n",
       " 356253                        NaN                NaN   \n",
       " 356254                     -373.0      164461.005719   \n",
       " \n",
       "                    ...               FLAG_CONT_MOBILE  FLAG_DOCUMENT_5  \\\n",
       " 0                  ...                              1                0   \n",
       " 1                  ...                              1                0   \n",
       " 2                  ...                              1                0   \n",
       " 3                  ...                              1                0   \n",
       " 4                  ...                              1                0   \n",
       " 5                  ...                              1                0   \n",
       " 6                  ...                              1                0   \n",
       " 7                  ...                              1                0   \n",
       " 8                  ...                              1                0   \n",
       " 9                  ...                              1                0   \n",
       " 10                 ...                              1                0   \n",
       " 11                 ...                              1                0   \n",
       " 12                 ...                              1                0   \n",
       " 13                 ...                              1                0   \n",
       " 14                 ...                              1                0   \n",
       " 15                 ...                              1                0   \n",
       " 16                 ...                              1                0   \n",
       " 17                 ...                              1                0   \n",
       " 18                 ...                              1                0   \n",
       " 19                 ...                              1                0   \n",
       " 20                 ...                              1                0   \n",
       " 21                 ...                              1                0   \n",
       " 22                 ...                              1                0   \n",
       " 23                 ...                              1                0   \n",
       " 24                 ...                              1                0   \n",
       " 25                 ...                              1                0   \n",
       " 26                 ...                              1                0   \n",
       " 27                 ...                              1                0   \n",
       " 28                 ...                              1                0   \n",
       " 29                 ...                              1                0   \n",
       " ...                ...                            ...              ...   \n",
       " 356225             ...                              1                0   \n",
       " 356226             ...                              1                0   \n",
       " 356227             ...                              1                0   \n",
       " 356228             ...                              1                0   \n",
       " 356229             ...                              1                0   \n",
       " 356230             ...                              1                0   \n",
       " 356231             ...                              1                0   \n",
       " 356232             ...                              1                0   \n",
       " 356233             ...                              1                0   \n",
       " 356234             ...                              1                0   \n",
       " 356235             ...                              1                0   \n",
       " 356236             ...                              1                0   \n",
       " 356237             ...                              1                0   \n",
       " 356238             ...                              1                0   \n",
       " 356239             ...                              1                0   \n",
       " 356240             ...                              1                0   \n",
       " 356241             ...                              1                0   \n",
       " 356242             ...                              1                0   \n",
       " 356243             ...                              1                0   \n",
       " 356244             ...                              1                0   \n",
       " 356245             ...                              1                0   \n",
       " 356246             ...                              1                0   \n",
       " 356247             ...                              1                0   \n",
       " 356248             ...                              1                0   \n",
       " 356249             ...                              1                0   \n",
       " 356250             ...                              1                1   \n",
       " 356251             ...                              1                0   \n",
       " 356252             ...                              1                0   \n",
       " 356253             ...                              1                0   \n",
       " 356254             ...                              1                0   \n",
       " \n",
       "         RATE_INTEREST_PRIMARY  AMT_CREDIT_SUM_DEBT_BMIN  SK_ID_PREV_CCMEAN  \\\n",
       " 0                         NaN                      0.00                NaN   \n",
       " 1                         NaN                      0.00                NaN   \n",
       " 2                         NaN                      0.00                NaN   \n",
       " 3                         NaN                       NaN          1489396.0   \n",
       " 4                         NaN                      0.00                NaN   \n",
       " 5                         NaN                      0.00                NaN   \n",
       " 6                         NaN                      0.00                NaN   \n",
       " 7                         NaN                      0.00                NaN   \n",
       " 8                         NaN                      0.00          1843384.0   \n",
       " 9                         NaN                       NaN                NaN   \n",
       " 10                        NaN                      0.00                NaN   \n",
       " 11                        NaN                      0.00                NaN   \n",
       " 12                        NaN                      0.00                NaN   \n",
       " 13                        NaN                      0.00                NaN   \n",
       " 14                        NaN                       NaN                NaN   \n",
       " 15                        NaN                      0.00                NaN   \n",
       " 16                        NaN                      0.00                NaN   \n",
       " 17                        NaN                       NaN          2594025.0   \n",
       " 18                        NaN                 205276.50                NaN   \n",
       " 19                        NaN                      0.00          1499902.0   \n",
       " 20                        NaN                       NaN                NaN   \n",
       " 21                        NaN                1886544.00                NaN   \n",
       " 22                        NaN                 935237.88                NaN   \n",
       " 23                        NaN                      0.00                NaN   \n",
       " 24                        NaN                      0.00                NaN   \n",
       " 25                        NaN                      0.00                NaN   \n",
       " 26                        NaN                      0.00                NaN   \n",
       " 27                        NaN                      0.00                NaN   \n",
       " 28                        NaN                      0.00                NaN   \n",
       " 29                        NaN                       NaN                NaN   \n",
       " ...                       ...                       ...                ...   \n",
       " 356225                    NaN                      0.00                NaN   \n",
       " 356226                    NaN                      0.00                NaN   \n",
       " 356227                    NaN                       NaN                NaN   \n",
       " 356228                    NaN                      0.00                NaN   \n",
       " 356229                    NaN                      0.00                NaN   \n",
       " 356230                    NaN                      0.00          2539206.0   \n",
       " 356231                    NaN                      0.00                NaN   \n",
       " 356232                    NaN                      0.00                NaN   \n",
       " 356233                    NaN                      0.00                NaN   \n",
       " 356234                    NaN                      0.00                NaN   \n",
       " 356235                    NaN                      0.00                NaN   \n",
       " 356236                    NaN                      0.00          1257709.0   \n",
       " 356237                    NaN                      0.00          1716212.0   \n",
       " 356238                    NaN                      0.00                NaN   \n",
       " 356239                    NaN                      0.00          1922609.0   \n",
       " 356240                    NaN                       NaN                NaN   \n",
       " 356241                    NaN                      0.00                NaN   \n",
       " 356242                    NaN                      0.00                NaN   \n",
       " 356243                    NaN                      0.00                NaN   \n",
       " 356244                    NaN                      0.00                NaN   \n",
       " 356245                    NaN                      0.00          2220268.0   \n",
       " 356246                    NaN                       NaN                NaN   \n",
       " 356247                    NaN                      0.00                NaN   \n",
       " 356248                    NaN                      0.00          1130837.0   \n",
       " 356249                    NaN                      0.00                NaN   \n",
       " 356250                    NaN                      0.00                NaN   \n",
       " 356251                    NaN                       NaN                NaN   \n",
       " 356252                    NaN                      0.00                NaN   \n",
       " 356253                    NaN                      0.00                NaN   \n",
       " 356254                    NaN                      0.00          1794451.0   \n",
       " \n",
       "         FLAG_DOCUMENT_20  NAME_PRODUCT_TYPE_BAVG  CNT_CREDIT_PROLONG_BMIN  \\\n",
       " 0                      0                     0.0                      0.0   \n",
       " 1                      0                     0.0                      0.0   \n",
       " 2                      0                     0.0                      0.0   \n",
       " 3                      0                     0.0                      NaN   \n",
       " 4                      0                     2.0                      0.0   \n",
       " 5                      0                     0.0                      0.0   \n",
       " 6                      0                     0.0                      0.0   \n",
       " 7                      0                     0.0                      0.0   \n",
       " 8                      0                     0.0                      0.0   \n",
       " 9                      0                     0.0                      NaN   \n",
       " 10                     0                     0.0                      0.0   \n",
       " 11                     0                     0.0                      0.0   \n",
       " 12                     0                     0.0                      0.0   \n",
       " 13                     0                     0.0                      0.0   \n",
       " 14                     0                     0.0                      NaN   \n",
       " 15                     0                     0.0                      0.0   \n",
       " 16                     0                     0.0                      0.0   \n",
       " 17                     0                     0.0                      NaN   \n",
       " 18                     0                     0.0                      0.0   \n",
       " 19                     0                     0.0                      0.0   \n",
       " 20                     0                     NaN                      NaN   \n",
       " 21                     0                     0.0                      0.0   \n",
       " 22                     0                     0.0                      0.0   \n",
       " 23                     0                     0.0                      0.0   \n",
       " 24                     0                     0.0                      0.0   \n",
       " 25                     0                     0.0                      0.0   \n",
       " 26                     0                     NaN                      0.0   \n",
       " 27                     0                     0.0                      0.0   \n",
       " 28                     0                     0.0                      0.0   \n",
       " 29                     0                     0.0                      NaN   \n",
       " ...                  ...                     ...                      ...   \n",
       " 356225                 0                     0.0                      0.0   \n",
       " 356226                 0                     0.0                      0.0   \n",
       " 356227                 0                     0.0                      NaN   \n",
       " 356228                 0                     0.0                      0.0   \n",
       " 356229                 0                     0.0                      0.0   \n",
       " 356230                 0                     0.0                      0.0   \n",
       " 356231                 0                     0.0                      0.0   \n",
       " 356232                 0                     0.0                      0.0   \n",
       " 356233                 0                     0.0                      0.0   \n",
       " 356234                 0                     0.0                      0.0   \n",
       " 356235                 0                     0.0                      0.0   \n",
       " 356236                 0                     0.0                      0.0   \n",
       " 356237                 0                     0.0                      0.0   \n",
       " 356238                 0                     0.0                      0.0   \n",
       " 356239                 0                     2.0                      0.0   \n",
       " 356240                 0                     0.0                      NaN   \n",
       " 356241                 0                     NaN                      0.0   \n",
       " 356242                 0                     0.0                      0.0   \n",
       " 356243                 0                     0.0                      0.0   \n",
       " 356244                 0                     0.0                      0.0   \n",
       " 356245                 0                     0.0                      0.0   \n",
       " 356246                 0                     0.0                      NaN   \n",
       " 356247                 0                     0.0                      0.0   \n",
       " 356248                 0                     0.0                      0.0   \n",
       " 356249                 0                     0.0                      0.0   \n",
       " 356250                 0                     1.0                      0.0   \n",
       " 356251                 0                     0.0                      NaN   \n",
       " 356252                 0                     0.0                      0.0   \n",
       " 356253                 0                     0.0                      0.0   \n",
       " 356254                 0                     0.0                      0.0   \n",
       " \n",
       "         RATE_INTEREST_PRIMARY_PRVMIN  AMT_CREDIT_SUM_OVERDUE_BMIN  \n",
       " 0                                NaN                          0.0  \n",
       " 1                                NaN                          0.0  \n",
       " 2                                NaN                          0.0  \n",
       " 3                                NaN                          NaN  \n",
       " 4                                NaN                          0.0  \n",
       " 5                                NaN                          0.0  \n",
       " 6                                NaN                          0.0  \n",
       " 7                                NaN                          0.0  \n",
       " 8                                NaN                          0.0  \n",
       " 9                                NaN                          NaN  \n",
       " 10                               NaN                          0.0  \n",
       " 11                               NaN                          0.0  \n",
       " 12                               NaN                          0.0  \n",
       " 13                               NaN                          0.0  \n",
       " 14                               NaN                          NaN  \n",
       " 15                               NaN                          0.0  \n",
       " 16                               NaN                          0.0  \n",
       " 17                               NaN                          NaN  \n",
       " 18                               NaN                          0.0  \n",
       " 19                               NaN                          0.0  \n",
       " 20                               NaN                          NaN  \n",
       " 21                               NaN                          0.0  \n",
       " 22                               NaN                          0.0  \n",
       " 23                               NaN                          0.0  \n",
       " 24                               NaN                          0.0  \n",
       " 25                               NaN                          0.0  \n",
       " 26                               NaN                          0.0  \n",
       " 27                               NaN                          0.0  \n",
       " 28                               NaN                          0.0  \n",
       " 29                               NaN                          NaN  \n",
       " ...                              ...                          ...  \n",
       " 356225                           NaN                          0.0  \n",
       " 356226                           NaN                          0.0  \n",
       " 356227                           NaN                          NaN  \n",
       " 356228                           NaN                          0.0  \n",
       " 356229                           NaN                          0.0  \n",
       " 356230                           NaN                          0.0  \n",
       " 356231                           NaN                          0.0  \n",
       " 356232                           NaN                          0.0  \n",
       " 356233                           NaN                          0.0  \n",
       " 356234                           NaN                          0.0  \n",
       " 356235                           NaN                          0.0  \n",
       " 356236                           NaN                          0.0  \n",
       " 356237                           NaN                          0.0  \n",
       " 356238                           NaN                          0.0  \n",
       " 356239                           NaN                          0.0  \n",
       " 356240                           NaN                          NaN  \n",
       " 356241                           NaN                          0.0  \n",
       " 356242                           NaN                          0.0  \n",
       " 356243                           NaN                          0.0  \n",
       " 356244                           NaN                          0.0  \n",
       " 356245                           NaN                          0.0  \n",
       " 356246                           NaN                          NaN  \n",
       " 356247                           NaN                          0.0  \n",
       " 356248                           NaN                          0.0  \n",
       " 356249                           NaN                          0.0  \n",
       " 356250                           NaN                          0.0  \n",
       " 356251                           NaN                          NaN  \n",
       " 356252                           NaN                          0.0  \n",
       " 356253                           NaN                          0.0  \n",
       " 356254                           NaN                          0.0  \n",
       " \n",
       " [356255 rows x 298 columns], [], None)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process_dataframe(new_high_corr_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get dummies\n",
    "merged_df = pd.get_dummies(new_high_corr_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "len_train = len(application_train)\n",
    "app_train = new_high_corr_data[:len_train]\n",
    "app_test = new_high_corr_data[len_train:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a new dataframe for polynomial features\n",
    "poly_features = app_train[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3', 'DAYS_BIRTH', 'TARGET']]\n",
    "poly_features_test = app_test[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3', 'DAYS_BIRTH']]\n",
    "\n",
    "# imputer for handling missing values\n",
    "from sklearn.preprocessing import Imputer\n",
    "imputer = Imputer(strategy = 'median')\n",
    "\n",
    "poly_target = poly_features['TARGET']\n",
    "\n",
    "poly_features = poly_features.drop(columns = ['TARGET'])\n",
    "\n",
    "# Need to impute missing values\n",
    "poly_features = imputer.fit_transform(poly_features)\n",
    "poly_features_test = imputer.transform(poly_features_test)\n",
    "\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "                                  \n",
    "# Create the polynomial object with specified degree\n",
    "poly_transformer = PolynomialFeatures(degree = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Polynomial Features shape:  (307511, 35)\n"
     ]
    }
   ],
   "source": [
    "poly_transformer.fit(poly_features)\n",
    "\n",
    "# Transform the features\n",
    "poly_features = poly_transformer.transform(poly_features)\n",
    "poly_features_test = poly_transformer.transform(poly_features_test)\n",
    "print('Polynomial Features shape: ', poly_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1',\n",
       " 'EXT_SOURCE_1',\n",
       " 'EXT_SOURCE_2',\n",
       " 'EXT_SOURCE_3',\n",
       " 'DAYS_BIRTH',\n",
       " 'EXT_SOURCE_1^2',\n",
       " 'EXT_SOURCE_1 EXT_SOURCE_2',\n",
       " 'EXT_SOURCE_1 EXT_SOURCE_3',\n",
       " 'EXT_SOURCE_1 DAYS_BIRTH',\n",
       " 'EXT_SOURCE_2^2',\n",
       " 'EXT_SOURCE_2 EXT_SOURCE_3',\n",
       " 'EXT_SOURCE_2 DAYS_BIRTH',\n",
       " 'EXT_SOURCE_3^2',\n",
       " 'EXT_SOURCE_3 DAYS_BIRTH',\n",
       " 'DAYS_BIRTH^2']"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "poly_transformer.get_feature_names(input_features = ['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3', 'DAYS_BIRTH'])[:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EXT_SOURCE_2 EXT_SOURCE_3                -0.193939\n",
      "EXT_SOURCE_1 EXT_SOURCE_2 EXT_SOURCE_3   -0.189605\n",
      "EXT_SOURCE_2^2 EXT_SOURCE_3              -0.176428\n",
      "EXT_SOURCE_2 EXT_SOURCE_3^2              -0.172282\n",
      "EXT_SOURCE_1 EXT_SOURCE_2                -0.166625\n",
      "EXT_SOURCE_1 EXT_SOURCE_3                -0.164065\n",
      "EXT_SOURCE_2                             -0.160295\n",
      "EXT_SOURCE_1 EXT_SOURCE_2^2              -0.156867\n",
      "EXT_SOURCE_3                             -0.155892\n",
      "EXT_SOURCE_1 EXT_SOURCE_3^2              -0.150822\n",
      "Name: TARGET, dtype: float64\n",
      "EXT_SOURCE_1 EXT_SOURCE_2 DAYS_BIRTH    0.155891\n",
      "EXT_SOURCE_2 DAYS_BIRTH                 0.156873\n",
      "EXT_SOURCE_2 EXT_SOURCE_3 DAYS_BIRTH    0.181283\n",
      "TARGET                                  1.000000\n",
      "1                                            NaN\n",
      "Name: TARGET, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# Create a dataframe of the features \n",
    "poly_features = pd.DataFrame(poly_features, \n",
    "                             columns = poly_transformer.get_feature_names(['EXT_SOURCE_1', 'EXT_SOURCE_2', \n",
    "                                                                           'EXT_SOURCE_3', 'DAYS_BIRTH']))\n",
    "\n",
    "# Add in the target\n",
    "poly_features['TARGET'] = poly_target\n",
    "\n",
    "# Find the correlations with the target\n",
    "poly_corrs = poly_features.corr()['TARGET'].sort_values()\n",
    "\n",
    "# Display most negative and most positive\n",
    "print(poly_corrs.head(10))\n",
    "print(poly_corrs.tail(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data with polynomial features shape:  (307511, 332)\n",
      "Testing data with polynomial features shape:   (48744, 332)\n"
     ]
    }
   ],
   "source": [
    "# Put test features into dataframe\n",
    "poly_features_test = pd.DataFrame(poly_features_test, \n",
    "                                  columns = poly_transformer.get_feature_names(['EXT_SOURCE_1', 'EXT_SOURCE_2', \n",
    "                                                                                'EXT_SOURCE_3', 'DAYS_BIRTH']))\n",
    "\n",
    "# Merge polynomial features into training dataframe\n",
    "poly_features['SK_ID_CURR'] = app_train['SK_ID_CURR']\n",
    "app_train_poly = app_train.merge(poly_features, on = 'SK_ID_CURR', how = 'left')\n",
    "\n",
    "# Merge polnomial features into testing dataframe\n",
    "poly_features_test['SK_ID_CURR'] = app_test['SK_ID_CURR']\n",
    "app_test_poly = app_test.merge(poly_features_test, on = 'SK_ID_CURR', how = 'left')\n",
    "\n",
    "# Align the dataframes\n",
    "app_train_poly, app_test_poly = app_train_poly.align(app_test_poly, join = 'inner', axis = 1)\n",
    "\n",
    "# Print out the new shapes\n",
    "print('Training data with polynomial features shape: ', app_train_poly.shape)\n",
    "print('Testing data with polynomial features shape:  ', app_test_poly.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(307511, 332)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app_train_poly.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data shape:  (307511, 332)\n",
      "Testing data shape:  (48744, 332)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler, Imputer\n",
    "\n",
    "# Drop the target from the training data\n",
    "if 'TARGET' in app_train_poly:\n",
    "    train = app_train_poly.drop(columns = ['TARGET'])\n",
    "else:\n",
    "    train = app_train_poly.copy()\n",
    "features = list(train.columns)\n",
    "\n",
    "# Copy of the testing data\n",
    "if 'TARGET' in app_test_poly:\n",
    "    test = app_test_poly.drop(columns = ['TARGET'])\n",
    "else:\n",
    "    test = app_test_poly.copy()\n",
    "\n",
    "# Median imputation of missing values\n",
    "imputer = Imputer(strategy = 'median')\n",
    "\n",
    "# Scale each feature to 0-1\n",
    "scaler = MinMaxScaler(feature_range = (0, 1))\n",
    "\n",
    "# Fit on the training data\n",
    "imputer.fit(train)\n",
    "\n",
    "# Transform both training and testing data\n",
    "train = imputer.transform(train)\n",
    "test = imputer.transform(test)\n",
    "\n",
    "# Repeat with the scaler\n",
    "scaler.fit(train)\n",
    "train = scaler.transform(train)\n",
    "test = scaler.transform(test)\n",
    "\n",
    "print('Training data shape: ', train.shape)\n",
    "print('Testing data shape: ', test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'numpy.ndarray' object has no attribute 'columns'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-77-0b73e5050d5d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'numpy.ndarray' object has no attribute 'columns'"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "77"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 AUC : 0.782187\n",
      "Fold 2 AUC : 0.780728\n",
      "Fold 3 AUC : 0.784836\n",
      "Fold 4 AUC : 0.784356\n",
      "Fold 5 AUC : 0.780832\n"
     ]
    }
   ],
   "source": [
    "# Format the training and testing data \n",
    "#train = np.array(train_df)\n",
    "#test = np.array(test_df.drop(columns='TARGET',inplace= True))\n",
    "\n",
    "#train_labels = train_df.pop('TARGET')\n",
    "\n",
    "# 10 fold cross validation\n",
    "folds = KFold(n_splits=5, shuffle=True, random_state=50)\n",
    "\n",
    "# Validation and test predictions\n",
    "valid_preds = np.zeros(train_df.shape[0])\n",
    "test_preds = np.zeros(test_df.shape[0])\n",
    "\n",
    "# Iterate through each fold\n",
    "for n_fold, (train_indices, valid_indices) in enumerate(folds.split(train)):\n",
    "    # Training data for the fold\n",
    "    train_fold, train_fold_labels = train[train_indices, :], labels[train_indices]\n",
    "    \n",
    "    # Validation data for the fold\n",
    "    valid_fold, valid_fold_labels = train[valid_indices, :], labels[valid_indices]\n",
    "    \n",
    "    # LightGBM classifier with hyperparameters\n",
    "    clf = LGBMClassifier(\n",
    "        n_estimators=10000,\n",
    "        learning_rate=0.1,\n",
    "        subsample=.8,\n",
    "        max_depth=-1,\n",
    "        reg_alpha=.1,\n",
    "        reg_lambda=.1,\n",
    "        min_split_gain=.01,\n",
    "        min_child_weight=2,\n",
    "        boosting = 'dart',\n",
    "        drop_rate = 0.02\n",
    "    )\n",
    "    \n",
    "    # Fit on the training data, evaluate on the validation data\n",
    "    clf.fit(train_fold, train_fold_labels, \n",
    "            eval_set= [(train_fold, train_fold_labels), (valid_fold, valid_fold_labels)], \n",
    "            eval_metric='auc', early_stopping_rounds=100, verbose = False\n",
    "           )\n",
    "    \n",
    "    # Validation preditions\n",
    "    valid_preds[valid_indices] = clf.predict_proba(valid_fold, num_iteration=clf.best_iteration_)[:, 1]\n",
    "    \n",
    "    # Testing predictions\n",
    "    test_preds += clf.predict_proba(test_df, num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits\n",
    "    \n",
    "    # Display the performance for the current fold\n",
    "    print('Fold %d AUC : %0.6f' % (n_fold + 1, roc_auc_score(valid_fold_labels, valid_preds[valid_indices])))\n",
    "    \n",
    "    # Delete variables to free up memory\n",
    "    del clf, train_fold, train_fold_labels, valid_fold, valid_fold_labels\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((307511, 806), (307511, 807))"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape,app_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = train\n",
    "y_train = app_train.TARGET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((307511, 806), (307511,))"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape,y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "partial_x_train_merged = x_train[:300000]\n",
    "x_val_merged = x_train[3000001:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "partial_y_train_merged = y_train[:300000]\n",
    "y_val_merged = y_train[300001:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import regularizers\n",
    "from keras.wrappers.scikit_learn import KerasClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 48min 21s, sys: 4min 9s, total: 52min 31s\n",
      "Wall time: 32min 5s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Function to create model, required for KerasClassifier\n",
    "def create_model(optimizer, init,reg):\n",
    "    model = models.Sequential()\n",
    "    model.add(layers.Dense(64, kernel_initializer=init, kernel_regularizer=regularizers.l2(reg),input_shape=(806,)))\n",
    "    model.add(layers.Dense(64, kernel_initializer=init, kernel_regularizer=regularizers.l2(reg),activation='relu'))\n",
    "    model.add(layers.Dense(1, kernel_initializer=init, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy',optimizer=optimizer, metrics=['accuracy'])\n",
    "    return(model)\n",
    "model = KerasClassifier(build_fn=create_model, verbose=0)\n",
    "\n",
    "# grid search epochs, batch size and optimizer\n",
    "optimizers = ['adam']\n",
    "init = ['normal']\n",
    "reg = [0.001]\n",
    "epochs = [20,25,50]\n",
    "batches = [200,400,500]\n",
    "\n",
    "param_grid = dict(optimizer = optimizers, init = init,epochs = epochs,batch_size = batches,reg =reg)\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, scoring='roc_auc')\n",
    "grid_result = grid.fit(partial_x_train_merged, partial_y_train_merged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.762180 using {'batch_size': 500, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam', 'reg': 0.001}\n",
      "0.761158 (0.003153) with: {'batch_size': 200, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam', 'reg': 0.001}\n",
      "0.760063 (0.003258) with: {'batch_size': 200, 'epochs': 25, 'init': 'normal', 'optimizer': 'adam', 'reg': 0.001}\n",
      "0.760901 (0.003037) with: {'batch_size': 200, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam', 'reg': 0.001}\n",
      "0.761003 (0.002995) with: {'batch_size': 400, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam', 'reg': 0.001}\n",
      "0.761618 (0.003116) with: {'batch_size': 400, 'epochs': 25, 'init': 'normal', 'optimizer': 'adam', 'reg': 0.001}\n",
      "0.761124 (0.003346) with: {'batch_size': 400, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam', 'reg': 0.001}\n",
      "0.761168 (0.003311) with: {'batch_size': 500, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam', 'reg': 0.001}\n",
      "0.761258 (0.003373) with: {'batch_size': 500, 'epochs': 25, 'init': 'normal', 'optimizer': 'adam', 'reg': 0.001}\n",
      "0.762180 (0.003618) with: {'batch_size': 500, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam', 'reg': 0.001}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "\tprint(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
